package spiders

import (
	"bytes"
	"compress/zlib"
	"dianping_sol/config"
	"dianping_sol/db"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

func updatePage(r io.Reader) (page int) {
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		return page
	}
	page = 1
	doc.Find(`div.page a`).Each(func(i int, selection *goquery.Selection) {
		pageStr := selection.Text()
		p, err := strconv.Atoi(pageStr)
		if err == nil {
			if p > page {
				page = p
			}
		}

	})
	return page
}

func update(initUrl string, grp *db.GRP) error {
	var t string
	if v, ok := MapC[grp.RName]; ok {
		t = v

	} else {
		t = "r"
	}
	req, err := http.NewRequest("GET", initUrl+"/g"+grp.G+t+grp.R+`o2`, nil)
	if err != nil {
		return err
	}

	req.Header.Set("User-Agent", config.Conf.DianPing.UserAgent)
	req.Header.Set("Cookie", config.Conf.DianPing.Cookie)
	req.Header.Set("Host", config.Conf.DianPing.Host)
	req.Header.Set("sec-ch-ua-platform", config.Conf.DianPing.Sec)
	req.Header.Set("Accept", config.Conf.DianPing.Accept)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == 200 {
		page := updatePage(resp.Body)
		//grp.P = strconv.Itoa(page)
		db.G_DB.Debug().Model(grp).Update(`p`, strconv.Itoa(page))
		return nil
	}
	return fmt.Errorf("request error %d", resp.StatusCode)
}

func StartWork(baseUrl string) error {

	//查询Message表最后一条数据,接着上一次继续跑
	var c int64

	var message db.Message
	err := db.G_DB.Debug().Model(&message).Count(&c).Error
	if err != nil {
		return err
	}
	count := int(c)
	var max struct{ Id int }
	err = db.G_DB.Debug().Model(&db.GRP{}).Select("MAX(id) as id").Scan(&max).Error
	if err != nil {
		return err
	}
	//没有记录从头开始
	if count == 0 {
		//查询最大gprs
		var max struct{ Id int }
		err := db.G_DB.Debug().Model(&db.GRP{}).Select("MAX(id) as id").Scan(&max).Error
		if err != nil {
			return err
		}
		for i := 1; i <= max.Id; i++ {
			log.Printf(`%d second break ... `, config.Conf.TimeOut.UrlTimeOut)
			time.Sleep(time.Duration(config.Conf.TimeOut.UrlTimeOut) * time.Second)
			var grp db.GRP
			db.G_DB.Where(`id=?`, i).Find(&grp)
			//更新页码
			err := update(baseUrl, &grp)
			if err != nil {
				panic(fmt.Errorf(`update error  %s ,selectId = %d,exit,`, err, i))
			}
			//处理页码
			ps := strings.TrimSpace(grp.P)
			p, _ := strconv.Atoi(ps)
			var countRom []int
		d:
			for j := 1; j <= p; j++ {
				log.Printf(`%d second break ... `, config.Conf.TimeOut.PageTimeOut)
				time.Sleep(time.Duration(config.Conf.TimeOut.PageTimeOut) * time.Second)
				grp.N = strconv.Itoa(j)
				db.G_DB.Debug().Model(&grp).Update(`n`, strconv.Itoa(j))
				count, _ := Deal(baseUrl, grp, 0)
				countRom = append(countRom, count)
				for i := 0; i < len(countRom)-1; i++ {
					if countRom[i] == 1 && countRom[i+1] == 1 {
						break d
					}
				}
			}

		}

	}

	if count > 0 {
		var m db.Message
		err = db.G_DB.Debug().Order(`id desc`).Limit(1).Find(&m).Error
		if err != nil {
			return err
		}
		//根据message获取grp的id
		var grbOld db.GRP
		err = db.G_DB.Debug().Where(`g=? and r=?`, m.G, m.R).Find(&grbOld).Error
		if err != nil {
			return err
		}
		n, _ := strconv.Atoi(m.N)
		p, _ := strconv.Atoi(grbOld.P)
		//l, _ := strconv.Atoi(m.L)
		var countRom []int
	c:
		for i := n; i <= p; i++ {
			grbOld.N = strconv.Itoa(i)
			db.G_DB.Debug().Model(&grbOld).Update(`n`, strconv.Itoa(i))
			log.Printf(`%d second break ... `, config.Conf.TimeOut.PageTimeOut)
			time.Sleep(time.Duration(config.Conf.TimeOut.PageTimeOut) * time.Second)
			count, _ := Deal(baseUrl, grbOld, 0)
			countRom = append(countRom, count)
			for i := 0; i < len(countRom)-1; i++ {
				if countRom[i] == 1 && countRom[i+1] == 1 {
					break c
				}
			}
		}

		for i := grbOld.Id + 1; i <= max.Id; i++ {
			var grp db.GRP
			db.G_DB.Where(`id=?`, i).Find(&grp)
			log.Printf(`%d second break ... `, config.Conf.TimeOut.UrlTimeOut)
			time.Sleep(time.Duration(config.Conf.TimeOut.UrlTimeOut) * time.Second)
			//更新页码
			err := update(baseUrl, &grp)
			if err != nil {
				log.Fatalf(`update error  %s ,selectId = %d,exit,`, err, i)
			}
			//处理页码
			ps := strings.TrimSpace(grp.P)
			p, _ := strconv.Atoi(ps)
			var countRom []int
		b:
			for j := 1; j <= p; j++ {
				grp.N = strconv.Itoa(j)
				db.G_DB.Debug().Model(&grp).Update(`n`, strconv.Itoa(j))
				log.Printf(`%d second break ... `, config.Conf.TimeOut.PageTimeOut)
				time.Sleep(time.Duration(config.Conf.TimeOut.PageTimeOut) * time.Second)
				count, _ := Deal(baseUrl, grp, 0)
				countRom = append(countRom, count)
				for i := 0; i < len(countRom)-1; i++ {
					if countRom[i] == 1 && countRom[i+1] == 1 {
						break b
					}
				}
			}

		}
		for i := 1; i <= max.Id; i++ {
			var grp db.GRP
			db.G_DB.Where(`id=?`, i).Find(&grp)
			//更新页码
			err := update(baseUrl, &grp)
			if err != nil {
				log.Fatalf(`update error  %s ,selectId = %d,exit,`, err, i)
			}
			log.Printf(`%d second break ... `, config.Conf.TimeOut.UrlTimeOut)
			time.Sleep(time.Duration(config.Conf.TimeOut.UrlTimeOut) * time.Second)
			//处理页码
			ps := strings.TrimSpace(grp.P)
			p, _ := strconv.Atoi(ps)
			var countRom []int
		a:
			for j := 1; j <= p; j++ {
				grp.N = strconv.Itoa(j)
				db.G_DB.Debug().Model(&grp).Update(`n`, strconv.Itoa(j))
				log.Printf(`%d second break ... `, config.Conf.TimeOut.PageTimeOut)
				time.Sleep(time.Duration(config.Conf.TimeOut.PageTimeOut) * time.Second)
				count, _ := Deal(baseUrl, grp, 0)
				countRom = append(countRom, count)
				for i := 0; i < len(countRom)-1; i++ {
					if countRom[i] == 1 && countRom[i+1] == 1 {
						break a
					}
				}
			}

		}
		StartWork(baseUrl)
	}

	return nil
}

// Deal L跳过前面多少个,遗留问题,现忽略,总是从当页第一个开始
func Deal(url string, grp db.GRP, L int) (int, error) {
	n, _ := strconv.Atoi(grp.N)
	var np string
	if n > 1 {
		np = `p` + strconv.Itoa(n)
	}
	var t string
	if v, ok := MapC[grp.RName]; ok {
		t = v

	} else {
		t = "r"
	}
	urls := url + "/g" + grp.G + t + grp.R + np + `o2`
	log.Println(`request url  `, urls)
	req, err := http.NewRequest("GET", urls, nil)
	if err != nil {
		return 0, err
	}
	req.Header.Set("User-Agent", config.Conf.DianPing.UserAgent)
	req.Header.Set("Cookie", config.Conf.DianPing.Cookie)
	req.Header.Set("Host", config.Conf.DianPing.Host)
	req.Header.Set("sec-ch-ua-platform", config.Conf.DianPing.Sec)
	req.Header.Set("Accept", config.Conf.DianPing.Accept)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return 0, err
	}
	defer resp.Body.Close()
	if resp.StatusCode == 200 {
		//r, _ := io.ReadAll(resp.Body)
		//fmt.Println(string(r))
		doc, err := goquery.NewDocumentFromReader(resp.Body)
		if err != nil {
			return 0, err
		}
		var messages []db.Message
		doc.Find(`#shop-all-list ul li`).Each(func(i int, selection *goquery.Selection) {
			var message db.Message
			message.G = grp.G
			message.R = grp.R
			message.N = grp.N
			pic, picHave := selection.Find(`div.pic a img`).Attr(`src`)
			if picHave {
				message.ShopImgUrl = pic
			}

			storeName := selection.Find(`.tit a h4`).Text()     //名称
			href, have := selection.Find(".tit a").Attr("href") //详细链接
			if have {
				message.ShopName = storeName
				message.ShopInfoUrl = href
				message.ShopId = href[strings.LastIndex(href, `/`)+1:]
			}
			comment := selection.Find(`.comment`).Find(`.review-num`).Text()
			comment = strings.ReplaceAll(comment, " ", "")
			comment = strings.ReplaceAll(comment, "\n", "") //评论
			re := regexp.MustCompile(`^(\d+)`)
			match := re.FindStringSubmatch(comment)
			if len(match) > 1 {
				num, err := strconv.Atoi(match[1])
				if err != nil {
					message.CommentCount = 0
				}
				message.CommentCount = num
			} else {
				message.CommentCount = 0
			}
			price := selection.Find(`.comment`).Find(`.mean-price`).Text()
			price = strings.ReplaceAll(price, " ", "")
			price = strings.ReplaceAll(price, "\n", "") //人均
			rePrice := regexp.MustCompile(`\d+`)
			matches := rePrice.FindAllString(price, -1)
			if len(matches) > 0 {
				num, err := strconv.ParseFloat(matches[0], 32)
				if err != nil {
					message.PriceAvg = 0
				}
				message.PriceAvg = num
			} else {
				message.PriceAvg = 0

			}
			selection.Find(`.tag-addr a`).Each(func(i int, selection *goquery.Selection) {
				if i == 0 {
					message.TagCuisine = selection.Text()
				}
				if i == 1 {
					message.TagLocation = selection.Text()
				}
			})
			var recommend []string
			selection.Find(`.recommend a`).Each(func(i int, selection *goquery.Selection) {
				recommend = append(recommend, strings.TrimSpace(selection.Text()))
			})
			if len(recommend) > 0 {
				var s string
				for _, v := range recommend {
					s += v + `,`
				}
				message.Recommend = s[:len(s)-1]
			}
			messages = append(messages, message)

		})
		//messages = messages[L:]
		count := 0
		for k, v := range messages {
			if v.CommentCount == 0 { //没有评价的商户直接跳过采集,增加速度
				log.Println(`没有评价的商户直接跳过采集 `, v)
				count++
				continue
			}
			if v.PriceAvg == 0 {
				log.Println(`没有均价的商户直接跳过采集 `, v)
				count++
				continue
			}
			v.N = grp.N
			v.L = strconv.Itoa(k + L + 1)
			log.Printf(`%d second break ... `, config.Conf.TimeOut.InfoTimeOut)
			time.Sleep(time.Duration(config.Conf.TimeOut.InfoTimeOut) * time.Second)
			if config.Conf.DianPing.Model == 1 {
				ModelOne(&v)
				v.Region = grp.RName
			} else {
				ShopInfo(&v)
				v.Region = grp.RName
			}
			//查询是否有v,如果有更新而不是插入
			var m db.Message
			db.G_DB.Debug().Where(`shop_id=?`, v.ShopId).Find(&m)
			if m.Id > 0 {
				v.Id = m.Id
				v.UpdatedAt = time.Now()
				v.CreatedAt = m.CreatedAt
				db.G_DB.Debug().Save(&v)
			} else {
				db.G_DB.Debug().Create(&v)
			}

		}
		if count == len(messages) { //如果一页的商户全都是无效商户,则返回1,外层循环如果遇见连续两次出现1,则跳过该选项
			return 1, nil
		}
	}
	return 0, nil
}

func ShopInfo(message *db.Message) error {
	req, err := http.NewRequest("GET", message.ShopInfoUrl, nil)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", config.Conf.DianPing.UserAgent)
	req.Header.Set("Cookie", config.Conf.DianPing.Cookie)
	req.Header.Set("Host", config.Conf.DianPing.Host)
	req.Header.Set("sec-ch-ua-platform", config.Conf.DianPing.Sec)
	req.Header.Set("Accept", config.Conf.DianPing.Accept)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == 200 {
		//>口味: 8.6 </span> <span class="item">环境: 8.8 </span> <span class="item">服务: 9.1
		doc, err := goquery.NewDocumentFromReader(resp.Body)
		if err != nil {
			return err
		}
		score := doc.Find(`#comment_score span`).Text()
		split := strings.Split(score, ` `)
		for k, v := range split {
			if v == "口味:" {
				num, err := strconv.ParseFloat(split[k+1], 32)
				if err != nil {
					message.ScoreKouwei = 0
				} else {
					message.ScoreKouwei = num
				}
			}
			if v == "环境:" {
				num, err := strconv.ParseFloat(split[k+1], 32)
				if err != nil {
					message.ScoreHuanjing = 0
				} else {
					message.ScoreHuanjing = num
				}
			}
			if v == "服务:" {
				num, err := strconv.ParseFloat(split[k+1], 32)
				if err != nil {
					message.ScoreFuwu = 0
				} else {
					message.ScoreFuwu = num
				}
			}
		}
		avg := (message.ScoreKouwei + message.ScoreHuanjing + message.ScoreFuwu) / 3
		strAvg := fmt.Sprintf("%.1f", avg)
		avg, _ = strconv.ParseFloat(strAvg, 32)
		message.ScoreAvg = avg

		addr := doc.Find(`#address`).Text()
		addr = strings.TrimSpace(addr)
		tel := doc.Find(`p.expand-info`).Text()
		idx := strings.Index(tel, "：")
		var p string
		if idx != -1 && len(tel) > 3 {
			tel = strings.TrimSpace(tel[idx+3:])
			i := strings.Split(tel, " ")
			for k, v := range i {
				s := strings.TrimSpace(v)
				re := regexp.MustCompile(`\d`)
				if re.MatchString(s) {
					if k == len(i)-1 {
						p += s
					} else {
						p += s + ","
					}
				}
			}
		} else {
			p = tel
		}

		shopBusinessHours := doc.Find(`p.info`).Find(`span.item`).Text()
		message.ShopBusinessHours = shopBusinessHours
		message.ShopTel = p
		message.ShopLocation = addr
		if message.ShopLocation == "" && message.ShopTel == "" { //如果两个都是空,我们认为
			log.Printf(`有可能我们请求商户具体信息有验证码了,所以我暂停5分钟,检查一下,你可以复制商户链接去浏览器验证一下,过一次验证码之后,重新更新cookie  %+v `, message)
			time.Sleep(5 * time.Minute)
			ShopInfo(message)
		}
	} else {
		log.Println(`请求详情发生错误,停止请求12小时 ...`)
		time.Sleep(12 * time.Hour)
		ShopInfo(message)
	}
	return nil
}

func ModelOne(message *db.Message) {
	score := "https://www.dianping.com/ajax/json/shopDynamic/reviewAndStar"
	info := "https://www.dianping.com/ajax/json/shopDynamic/basicHideInfo"
	token := getToken(message.ShopInfoUrl)
	requestScore := fmt.Sprintf(score+"?shopId=%s&cityId=%s&mainCategoryId=%s&_token=%s&uuid=%s&platform=1&partner=150&optimusCode=10&originUrl=%s",
		message.ShopId, "9", "2823", token, config.Conf.DianPing.UUid, message.ShopInfoUrl)
	requestInfo := fmt.Sprintf(info+"?shopId=%s&_token=%s&tcv=%s&uuid=%s&platform=1&partner=150&optimusCode=10&originUrl=%s",
		message.ShopId, token, config.Conf.DianPing.Tcv, config.Conf.DianPing.UUid, message.ShopInfoUrl)
	reqScore, err := http.NewRequest("GET", requestScore, nil)
	if err != nil {
		return
	}
	ajaxSetReq(reqScore)
	client1 := &http.Client{}
	resp1, err := client1.Do(reqScore)
	defer resp1.Body.Close()
	if resp1.StatusCode == 200 {
		r1, _ := io.ReadAll(resp1.Body)
		var resultScore struct {
			Msg                       string   `json:"msg"`
			ShopRefinedScoreValueList []string `json:"shopRefinedScoreValueList"`
			Code                      int      `json:"code"`
			AvgPriceTitle             string   `json:"avgPriceTitle"`
			AvgPrice                  string   `json:"avgPrice"`
			Score4                    int      `json:"score4"`
			DefaultReviewCount        string   `json:"defaultReviewCount"`
			FiveScore                 string   `json:"fiveScore"`
			ShopPower                 int      `json:"shopPower"`
			ShopScoreTitleList        []string `json:"shopScoreTitleList"`
		}
		err = json.Unmarshal(r1, &resultScore)
		if err != nil {
			log.Printf("解析返回结果错误,结果为 %s,无法解析", string(r1))
			return
		}
		if len(resultScore.ShopRefinedScoreValueList) == 3 {
			kouwei, err := strconv.ParseFloat(resultScore.ShopRefinedScoreValueList[0], 32)
			if err != nil {
				message.ScoreKouwei = 0
			}
			message.ScoreKouwei = kouwei
			huanjing, err := strconv.ParseFloat(resultScore.ShopRefinedScoreValueList[1], 32)
			if err != nil {
				message.ScoreHuanjing = 0
			}
			message.ScoreHuanjing = huanjing
			fuwu, err := strconv.ParseFloat(resultScore.ShopRefinedScoreValueList[2], 32)
			if err != nil {
				message.ScoreFuwu = 0
			}
			message.ScoreFuwu = fuwu
		}
	}

	reqInfo, err := http.NewRequest("GET", requestInfo, nil)
	if err != nil {
		return
	}
	ajaxSetReq(reqInfo)
	client2 := &http.Client{}
	resp2, err := client2.Do(reqInfo)
	defer resp2.Body.Close()
	if resp2.StatusCode == 200 {
		r2, _ := io.ReadAll(resp2.Body)
		var resultInfo struct {
			Msg struct {
				ShopInfo struct {
					ShopId        string `json:"shopId"`
					ShopType      int    `json:"shopType"`
					ShopName      string `json:"shopName"`
					BranchName    string `json:"branchName"`
					Address       string `json:"address"`
					CrossRoad     string `json:"crossRoad"`
					PhoneNo       string `json:"phoneNo"`
					PhoneNo2      string `json:"phoneNo2"`
					CityId        int    `json:"cityId"`
					ShopGroupId   string `json:"shopGroupId"`
					AltName       string `json:"altName"`
					Glat          string `json:"glat"`
					Glng          string `json:"glng"`
					BusinessHours string `json:"businessHours"`
				} `json:"shopInfo"`
				ParkInfo struct {
					ParkReviewCount int `json:"parkReviewCount"`
				} `json:"parkInfo"`
				Seo string `json:"seo"`
			} `json:"msg"`
			Code int `json:"code"`
		}
		err = json.Unmarshal(r2, &resultInfo)
		if err != nil {
			log.Printf("解析返回结果错误,结果为 %s,无法解析", string(r2))
			return
		}
		message.ShopLocation = resultInfo.Msg.ShopInfo.Address
		tel := resultInfo.Msg.ShopInfo.PhoneNo
		if strings.TrimSpace(resultInfo.Msg.ShopInfo.PhoneNo2) != "" {
			tel = tel + "," + strings.TrimSpace(resultInfo.Msg.ShopInfo.PhoneNo2)
		}
		message.ShopTel = tel
		avg := (message.ScoreKouwei + message.ScoreHuanjing + message.ScoreFuwu) / 3
		strAvg := fmt.Sprintf("%.1f", avg)
		avg, _ = strconv.ParseFloat(strAvg, 32)
		message.ScoreAvg = avg
	}
}

func ajaxSetReq(req *http.Request) {
	req.Header.Set("User-Agent", config.Conf.DianPing.UserAgent)
	req.Header.Set("Cookie", config.Conf.DianPing.Cookie)
	req.Header.Set("sec-ch-ua-platform", config.Conf.DianPing.Sec)
	req.Header.Set("Sec-Ch-Ua", config.Conf.DianPing.Ua)
	req.Header.Set("Accept", config.Conf.DianPing.AcceptAjax)
	req.Header.Set("X-Requested-With", config.Conf.DianPing.RequestWith)
}
func getToken(shopURL string) string {
	ts := time.Now().UnixNano() / int64(time.Millisecond)
	cts := time.Now().UnixNano()/int64(time.Millisecond) - 600
	brVD := []int{1920, 186}
	brR := []interface{}{
		[]int{1920, 1080},
		[]int{1920, 1040},
		24,
		24,
	}
	bI := []string{shopURL, shopURL}
	mT := []string{"1244,588"}
	sign := "eJxTKs7IL/BMsTU2NTAwMLVUAgApvgRP"

	data := map[string]interface{}{
		"rId":  "100041",
		"ver":  "1.0.6",
		"ts":   ts,
		"cts":  cts,
		"brVD": brVD,
		"brR":  brR,
		"bI":   bI,
		"mT":   mT,
		"kT":   []string{},
		"aT":   []string{},
		"tT":   []string{},
		"aM":   "",
		"sign": sign,
	}

	jsonBytes, err := json.Marshal(data)
	if err != nil {
		fmt.Println(err)
		return ""
	}

	var buf bytes.Buffer
	w := zlib.NewWriter(&buf)
	_, err = w.Write(jsonBytes)
	if err != nil {
		fmt.Println(err)
		return ""
	}
	err = w.Close()
	if err != nil {
		fmt.Println(err)
		return ""
	}

	token := base64.StdEncoding.EncodeToString(buf.Bytes())
	return token
}
